module Spider

  def self.included(base)
    base.extend Methods
    base.send :include, Methods
  end # self.included


  module Methods
    require 'open-uri'
    require 'nokogiri'
    require 'timeout'
    
    def open_page(cat_url, code = 'GBK', out_time = 60)
      Rails.logger.info "抓取目标链接: #{cat_url}"
      timeout(out_time) do
        agent = Thread.current[:agent]
        page = agent ? agent.get(cat_url).body : open(cat_url)
        doc = Nokogiri::HTML(page, nil, code)
#        task_result = TaskResult.new
#        task_result.source = doc.to_s
#        task_result.save!
        return doc
      end
    rescue Timeout::Error => e
      Rails.logger.warn "spider 连接超时, url: #{cat_url}"
      nil
    rescue Exception => e
      Rails.logger.warn "spider 连接错误, url: #{cat_url} #{e.message} \n #{e.backtrace.join("\n")}"
      nil
    end

    def strip_nbsp(str)
      return str if str.blank?
      str[0].to_i == 194 ? str[2..-1] : str
    end

  end

end
